package extractorCode;



import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
//import org.jsoup.*;


public class boing {
	
	private final static int INITIAL_BUF_SIZE = 65536;
	public static void main(String args[]) throws Exception
	{
	
	
		BufferedWriter wr; 
		int month,year,pageNo;
		
		for(int k=2008;k<=2010;k++)
		{
			for(int i=1;i<=12;i++)
			{
				wr= new BufferedWriter(new FileWriter("c:\\newData\\Boing\\"+k+"\\boing_urls"+i+"_"+k+".txt"));
				for(int l=0;l<=22;l++)
		{
				System.out.println(k+" "+i);
				pageNo=0;
				try{
		System.out.println("page"+(l));	
	
		String page = (fetchPage("http://boingboing.net/"+k+"/"+i+"/page/"+l));
		pageNo++;;
		//System.out.println(page);
		page= page.replaceAll(">\\s*<", "><");
		
		page = page.replaceAll("\t", "");
		page = page.replaceAll("\b", "");
		page = page.replaceAll("\r", "");
		page = page.replaceAll("\n", "");
		page = page.replaceAll("\r\n", "");

		Matcher newsItem = Pattern.compile("<h2 class=\"headline\"><a href=\"(.*?)\"", Pattern.CASE_INSENSITIVE).matcher( page );
	
	//	newsItem.find();

		
		
		
	while(newsItem.find())
		{
	//	System.out.println(newsItem.group(1));
		
		String text = newsItem.group(1);
		//System.out.println(text);
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		System.out.println(text);
		wr.write(text+"\n");
		//wr.write("\n");
		
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
				}
				catch(Exception e){e.printStackTrace(); 
				wr.flush();
				
				wr.close();
				break;
				}
		
		}
				wr.close();	
			}}

		
		}
	
	
	
	private static String fetchPage(String urlString){
		StringBuilder buffer = new StringBuilder(INITIAL_BUF_SIZE);
		try {
			URL url = new URL( urlString );
			  HttpURLConnection.setFollowRedirects(false);
              HttpURLConnection urlConn = (HttpURLConnection)url.openConnection();
              urlConn.setRequestProperty("Accept-Language", "en-us,en;q=0.5"); 
              urlConn.setRequestProperty("User-agent","Mozilla/2.0.0.11");
              urlConn.connect();
			BufferedInputStream reader = new BufferedInputStream(urlConn.getInputStream());
			int temp = reader.read();
			while(temp != -1){
				buffer.append((char)temp);
				temp = reader.read();
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return buffer.toString();
	}

}
